{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Serialization example\n",
    "\n",
    "This example illustrates how to load a dataset from JSON, run the model on that dataset and write the output back to JSON. At the end the example is also shown for `msgpack`."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load dependencies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from pandas import DataFrame\n",
    "import pprint\n",
    "\n",
    "from power_grid_model import PowerGridModel, ComponentType, ComponentAttributeFilterOptions\n",
    "from power_grid_model.utils import json_deserialize, json_serialize"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load a dataset from a JSON file\n",
    "\n",
    "The data is in the `data/serialized-input.json` file.\n",
    "\n",
    "### Load the JSON file\n",
    "\n",
    "This is just for illustration purposes."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'attributes': {'node': ['id', 'u_rated'],\n",
      "                'source': ['id', 'node', 'status', 'u_ref', 'sk'],\n",
      "                'sym_load': ['id',\n",
      "                             'node',\n",
      "                             'status',\n",
      "                             'type',\n",
      "                             'p_specified',\n",
      "                             'q_specified']},\n",
      " 'data': {'line': [{'c1': 4e-05,\n",
      "                    'from_node': 1,\n",
      "                    'from_status': 1,\n",
      "                    'i_n': 500.0,\n",
      "                    'id': 4,\n",
      "                    'r1': 0.11,\n",
      "                    'tan1': 0.1,\n",
      "                    'to_node': 2,\n",
      "                    'to_status': 1,\n",
      "                    'x1': 0.12},\n",
      "                   {'c1': 5e-05,\n",
      "                    'from_node': 2,\n",
      "                    'from_status': 1,\n",
      "                    'i_n': 550.0,\n",
      "                    'id': 5,\n",
      "                    'r1': 0.15,\n",
      "                    'tan1': 0.12,\n",
      "                    'to_node': 3,\n",
      "                    'to_status': 1,\n",
      "                    'x1': 0.16}],\n",
      "          'node': [[1, 10500.0], [2, 10500.0], [3, 10500.0]],\n",
      "          'source': [[15, 1, 1, 1.03, 1e+20],\n",
      "                     [16, 1, 1, 1.04, None],\n",
      "                     {'id': 17,\n",
      "                      'node': 1,\n",
      "                      'rx_ratio': 0.2,\n",
      "                      'sk': 10000000000.0,\n",
      "                      'status': 1,\n",
      "                      'u_ref': 1.03}],\n",
      "          'sym_load': [[7, 2, 1, 0, 1010000.0, 210000.0],\n",
      "                       [8, 3, 1, 0, 1020000.0, 220000.0]]},\n",
      " 'is_batch': False,\n",
      " 'type': 'input',\n",
      " 'version': '1.0'}\n"
     ]
    }
   ],
   "source": [
    "with open(\"data/serialized_input.json\") as fp:\n",
    "    data = fp.read()\n",
    "\n",
    "pprint.pprint(json.loads(data))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Deserialize the JSON data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "components: [<ComponentType.node: 'node'>, <ComponentType.line: 'line'>, <ComponentType.source: 'source'>, <ComponentType.sym_load: 'sym_load'>]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "array([(1, 10500.), (2, 10500.), (3, 10500.)],\n",
       "      dtype={'names': ['id', 'u_rated'], 'formats': ['<i4', '<f8'], 'offsets': [0, 8], 'itemsize': 16, 'aligned': True})"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>u_rated</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>10500.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>10500.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>10500.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  u_rated\n",
       "0   1  10500.0\n",
       "1   2  10500.0\n",
       "2   3  10500.0"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dataset = json_deserialize(data)\n",
    "\n",
    "print(\"components:\", list(dataset.keys()))\n",
    "display(dataset[ComponentType.node])\n",
    "display(DataFrame(dataset[ComponentType.node]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Run power flow calculation on the loaded input data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>energized</th>\n",
       "      <th>u_pu</th>\n",
       "      <th>u</th>\n",
       "      <th>u_angle</th>\n",
       "      <th>p</th>\n",
       "      <th>q</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1.030000</td>\n",
       "      <td>10815.000000</td>\n",
       "      <td>-2.530317e-14</td>\n",
       "      <td>2.408998e+06</td>\n",
       "      <td>-2.863495e+06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>1.029997</td>\n",
       "      <td>10814.968183</td>\n",
       "      <td>-4.398000e-03</td>\n",
       "      <td>-1.010000e+06</td>\n",
       "      <td>-2.100000e+05</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>1</td>\n",
       "      <td>1.029484</td>\n",
       "      <td>10809.581008</td>\n",
       "      <td>-6.839956e-03</td>\n",
       "      <td>-1.020000e+06</td>\n",
       "      <td>-2.200000e+05</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   id  energized      u_pu             u       u_angle             p  \\\n",
       "0   1          1  1.030000  10815.000000 -2.530317e-14  2.408998e+06   \n",
       "1   2          1  1.029997  10814.968183 -4.398000e-03 -1.010000e+06   \n",
       "2   3          1  1.029484  10809.581008 -6.839956e-03 -1.020000e+06   \n",
       "\n",
       "              q  \n",
       "0 -2.863495e+06  \n",
       "1 -2.100000e+05  \n",
       "2 -2.200000e+05  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model = PowerGridModel(dataset)\n",
    "output = model.calculate_power_flow()\n",
    "\n",
    "display(DataFrame(output[ComponentType.node]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Serialize the output dataset\n",
    "\n",
    "### Default format\n",
    "\n",
    "By default, the data is formatted nicely"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"version\": \"1.0\",\n",
      "  \"type\": \"sym_output\",\n",
      "  \"is_batch\": false,\n",
      "  \"attributes\": {},\n",
      "  \"data\": {\n",
      "    \"node\": [\n",
      "      {\"id\": 1, \"energized\": 1, \"u_pu\": 1.030000000001025, \"u\": 10815.00000001077, \"u_angle\": -2.530316910142707e-14, \"p\": 2408997.839438867, \"q\": -2863495.364674167},\n",
      "      {\"id\": 2, \"energized\": 1, \"u_pu\": 1.029996969815606, \"u\": 10814.96818306386, \"u_angle\": -0.004397999804754745, \"p\": -1009999.99999997, \"q\": -210000.0000000655},\n",
      "      {\"id\": 3, \"energized\": 1, \"u_pu\": 1.029483905569345, \"u\": 10809.58100847812, \"u_angle\": -0.006839956175380238, \"p\": -1019999.999999999, \"q\": -219999.9999999689}\n",
      "    ],\n",
      "    \"line\": [\n",
      "      {\"id\": 4, \"energized\": 1, \"loading\": 0.3995319091937107, \"p_from\": 2408997.839438867, \"q_from\": -2863495.364674167, \"i_from\": 199.7659545968554, \"s_from\": 3742041.7279784, \"p_to\": -2252625.764367544, \"q_to\": 1403928.536947823, \"i_to\": 141.6984332838951, \"s_to\": 2654305.591138465},\n",
      "      {\"id\": 5, \"energized\": 1, \"loading\": 0.1977047433812966, \"p_from\": 1242625.764367574, \"q_from\": -1613928.536947772, \"i_from\": 108.7376088597131, \"s_from\": 2036880.976553238, \"p_to\": -1019999.999999999, \"q_to\": -219999.9999999689, \"i_to\": 55.73199226981006, \"s_to\": 1043455.796859639}\n",
      "    ],\n",
      "    \"source\": [\n",
      "      {\"id\": 15, \"energized\": 1, \"p\": -7836685.751732428, \"q\": -105348495.343833, \"i\": 5639.485452974508, \"s\": 105639571.7275539, \"pf\": -0.07418324046166491},\n",
      "      {\"id\": 16, \"energized\": 1, \"p\": 10248883.05839198, \"q\": 102488830.5811954, \"i\": 5498.573991718361, \"s\": 102999999.9895415, \"pf\": 0.09950371902361781},\n",
      "      {\"id\": 17, \"energized\": 1, \"p\": -0.001807829591278542, \"q\": -0.0104081191056169, \"i\": 5.639485452974507e-07, \"s\": 0.01056395717275539, \"pf\": -0.1711318553942043}\n",
      "    ],\n",
      "    \"sym_load\": [\n",
      "      {\"id\": 7, \"energized\": 1, \"p\": 1010000, \"q\": 210000, \"i\": 55.07135393955915, \"s\": 1031600.697944704, \"pf\": 0.979060989404389},\n",
      "      {\"id\": 8, \"energized\": 1, \"p\": 1020000, \"q\": 220000, \"i\": 55.73199226981046, \"s\": 1043455.796859647, \"pf\": 0.9775210440823288}\n",
      "    ]\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "serialized_output = json_serialize(output)\n",
    "\n",
    "print(serialized_output)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Compact serialization\n",
    "\n",
    "In the full result, all attributes are explicitly listed for each component.\n",
    "In addition, all attributes are listed on a separate whiteline.\n",
    "This is fairly expensive memory-wise.\n",
    "\n",
    "If you need a more memory-efficient output, you can tell the serializer to use compact lists and to avoid using redundant newlines and whitespaces."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"version\":\"1.0\",\"type\":\"sym_output\",\"is_batch\":false,\"attributes\":{\"node\":[\"id\",\"energized\",\"u_pu\",\"u\",\"u_angle\",\"p\",\"q\"],\"line\":[\"id\",\"energized\",\"loading\",\"p_from\",\"q_from\",\"i_from\",\"s_from\",\"p_to\",\"q_to\",\"i_to\",\"s_to\"],\"source\":[\"id\",\"energized\",\"p\",\"q\",\"i\",\"s\",\"pf\"],\"sym_load\":[\"id\",\"energized\",\"p\",\"q\",\"i\",\"s\",\"pf\"]},\"data\":{\"node\":[[1,1,1.030000000001025,10815.00000001077,-2.530316910142707e-14,2408997.839438867,-2863495.364674167],[2,1,1.029996969815606,10814.96818306386,-0.004397999804754745,-1009999.99999997,-210000.0000000655],[3,1,1.029483905569345,10809.58100847812,-0.006839956175380238,-1019999.999999999,-219999.9999999689]],\"line\":[[4,1,0.3995319091937107,2408997.839438867,-2863495.364674167,199.7659545968554,3742041.7279784,-2252625.764367544,1403928.536947823,141.6984332838951,2654305.591138465],[5,1,0.1977047433812966,1242625.764367574,-1613928.536947772,108.7376088597131,2036880.976553238,-1019999.999999999,-219999.9999999689,55.73199226981006,1043455.796859639]],\"source\":[[15,1,-7836685.751732428,-105348495.343833,5639.485452974508,105639571.7275539,-0.07418324046166491],[16,1,10248883.05839198,102488830.5811954,5498.573991718361,102999999.9895415,0.09950371902361781],[17,1,-0.001807829591278542,-0.0104081191056169,5.639485452974507e-07,0.01056395717275539,-0.1711318553942043]],\"sym_load\":[[7,1,1010000,210000,55.07135393955915,1031600.697944704,0.979060989404389],[8,1,1020000,220000,55.73199226981046,1043455.796859647,0.9775210440823288]]}}\n"
     ]
    }
   ],
   "source": [
    "serialized_output = json_serialize(output, use_compact_list=True, indent=-1)\n",
    "\n",
    "print(serialized_output)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The compact result is still valid JSON"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'attributes': {'line': ['id',\n",
      "                         'energized',\n",
      "                         'loading',\n",
      "                         'p_from',\n",
      "                         'q_from',\n",
      "                         'i_from',\n",
      "                         's_from',\n",
      "                         'p_to',\n",
      "                         'q_to',\n",
      "                         'i_to',\n",
      "                         's_to'],\n",
      "                'node': ['id', 'energized', 'u_pu', 'u', 'u_angle', 'p', 'q'],\n",
      "                'source': ['id', 'energized', 'p', 'q', 'i', 's', 'pf'],\n",
      "                'sym_load': ['id', 'energized', 'p', 'q', 'i', 's', 'pf']},\n",
      " 'data': {'line': [[4,\n",
      "                    1,\n",
      "                    0.3995319091937107,\n",
      "                    2408997.839438867,\n",
      "                    -2863495.364674167,\n",
      "                    199.7659545968554,\n",
      "                    3742041.7279784,\n",
      "                    -2252625.764367544,\n",
      "                    1403928.536947823,\n",
      "                    141.6984332838951,\n",
      "                    2654305.591138465],\n",
      "                   [5,\n",
      "                    1,\n",
      "                    0.1977047433812966,\n",
      "                    1242625.764367574,\n",
      "                    -1613928.536947772,\n",
      "                    108.7376088597131,\n",
      "                    2036880.976553238,\n",
      "                    -1019999.999999999,\n",
      "                    -219999.9999999689,\n",
      "                    55.73199226981006,\n",
      "                    1043455.796859639]],\n",
      "          'node': [[1,\n",
      "                    1,\n",
      "                    1.030000000001025,\n",
      "                    10815.00000001077,\n",
      "                    -2.530316910142707e-14,\n",
      "                    2408997.839438867,\n",
      "                    -2863495.364674167],\n",
      "                   [2,\n",
      "                    1,\n",
      "                    1.029996969815606,\n",
      "                    10814.96818306386,\n",
      "                    -0.004397999804754745,\n",
      "                    -1009999.99999997,\n",
      "                    -210000.0000000655],\n",
      "                   [3,\n",
      "                    1,\n",
      "                    1.029483905569345,\n",
      "                    10809.58100847812,\n",
      "                    -0.006839956175380238,\n",
      "                    -1019999.999999999,\n",
      "                    -219999.9999999689]],\n",
      "          'source': [[15,\n",
      "                      1,\n",
      "                      -7836685.751732428,\n",
      "                      -105348495.343833,\n",
      "                      5639.485452974508,\n",
      "                      105639571.7275539,\n",
      "                      -0.07418324046166491],\n",
      "                     [16,\n",
      "                      1,\n",
      "                      10248883.05839198,\n",
      "                      102488830.5811954,\n",
      "                      5498.573991718361,\n",
      "                      102999999.9895415,\n",
      "                      0.0995037190236178],\n",
      "                     [17,\n",
      "                      1,\n",
      "                      -0.001807829591278542,\n",
      "                      -0.0104081191056169,\n",
      "                      5.639485452974507e-07,\n",
      "                      0.01056395717275539,\n",
      "                      -0.1711318553942043]],\n",
      "          'sym_load': [[7,\n",
      "                        1,\n",
      "                        1010000,\n",
      "                        210000,\n",
      "                        55.07135393955915,\n",
      "                        1031600.697944704,\n",
      "                        0.979060989404389],\n",
      "                       [8,\n",
      "                        1,\n",
      "                        1020000,\n",
      "                        220000,\n",
      "                        55.73199226981046,\n",
      "                        1043455.796859647,\n",
      "                        0.9775210440823288]]},\n",
      " 'is_batch': False,\n",
      " 'type': 'sym_output',\n",
      " 'version': '1.0'}\n"
     ]
    }
   ],
   "source": [
    "pprint.pprint(json.loads(serialized_output))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Msgpack serialization\n",
    "\n",
    "To have even higher performance and smaller space, you can use the binary format [`msgpack`](https://msgpack.org/). The example below shows a round trip to dump and load `msgpack` data, and intantiate model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Serialize to msgpack\n",
    "\n",
    "We can serialize the output data into `msgpack` binary with and without compact list. The result is a `bytes` object. See the resulted differences in length of the data. The differences will be significant when you have a large dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Type of the returned objects: <class 'bytes'>, <class 'bytes'>\n",
      "Length of not-compact data: 977\n",
      "Length of compact data: 802\n"
     ]
    }
   ],
   "source": [
    "from power_grid_model.utils import msgpack_serialize\n",
    "\n",
    "msgpack_data_not_compact = msgpack_serialize(output, use_compact_list=False)\n",
    "msgpack_data_compact = msgpack_serialize(output, use_compact_list=True)\n",
    "\n",
    "print(f\"Type of the returned objects: {type(msgpack_data_not_compact)}, {type(msgpack_data_compact)}\")\n",
    "print(f\"Length of not-compact data: {len(msgpack_data_not_compact)}\")\n",
    "print(f\"Length of compact data: {len(msgpack_data_compact)}\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Deserialize from msgpack\n",
    "\n",
    "We can deserialize the data we just created. We then print the node result. Note that they are exactly the same."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----Node result from not compact data----\n",
      "   id  energized      u_pu             u       u_angle             p  \\\n",
      "0   1          1  1.030000  10815.000000 -2.530317e-14  2.408998e+06   \n",
      "1   2          1  1.029997  10814.968183 -4.398000e-03 -1.010000e+06   \n",
      "2   3          1  1.029484  10809.581008 -6.839956e-03 -1.020000e+06   \n",
      "\n",
      "              q  \n",
      "0 -2.863495e+06  \n",
      "1 -2.100000e+05  \n",
      "2 -2.200000e+05  \n",
      "----Node result from compact data----\n",
      "   id  energized      u_pu             u       u_angle             p  \\\n",
      "0   1          1  1.030000  10815.000000 -2.530317e-14  2.408998e+06   \n",
      "1   2          1  1.029997  10814.968183 -4.398000e-03 -1.010000e+06   \n",
      "2   3          1  1.029484  10809.581008 -6.839956e-03 -1.020000e+06   \n",
      "\n",
      "              q  \n",
      "0 -2.863495e+06  \n",
      "1 -2.100000e+05  \n",
      "2 -2.200000e+05  \n"
     ]
    }
   ],
   "source": [
    "from power_grid_model.utils import msgpack_deserialize\n",
    "\n",
    "output_data_not_compact = msgpack_deserialize(msgpack_data_not_compact)\n",
    "output_data_compact = msgpack_deserialize(msgpack_data_compact)\n",
    "\n",
    "print(\"----Node result from not compact data----\")\n",
    "print(DataFrame(output_data_not_compact[ComponentType.node]))\n",
    "print(\"----Node result from compact data----\")\n",
    "print(DataFrame(output_data_compact[ComponentType.node]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Selective deserialization and dataset format\n",
    "\n",
    "To control the dataset returned by the deserialization functionality, you can use the `data_filter` argument.\n",
    "\n",
    "### Deserialization to columnar dataset\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "components: [<ComponentType.node: 'node'>, <ComponentType.line: 'line'>, <ComponentType.source: 'source'>, <ComponentType.sym_load: 'sym_load'>]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "{'id': array([1, 2, 3], dtype=int32),\n",
       " 'u_rated': array([10500., 10500., 10500.])}"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dataset = json_deserialize(data, data_filter=ComponentAttributeFilterOptions.everything)\n",
    "\n",
    "print(\"components:\", list(dataset.keys()))\n",
    "display(dataset[ComponentType.node])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Deserialized data format selection per component type\n",
    "\n",
    "To select specific components and data formats for the deserialized data, provide a dictionary of components and their desired output types to the `data_filter`."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "components: [<ComponentType.node: 'node'>, <ComponentType.line: 'line'>, <ComponentType.source: 'source'>, <ComponentType.sym_load: 'sym_load'>]\n",
      "node attributes: ['id', 'u_rated']\n",
      "source attributes: ['id', 'node', 'status', 'u_ref', 'sk']\n",
      "sym_load attributes: ['id', 'node', 'status', 'type', 'p_specified', 'q_specified']\n",
      "line attributes: ['id', 'from_node', 'to_node', 'from_status', 'to_status', 'r1', 'x1', 'c1', 'tan1', 'i_n']\n"
     ]
    }
   ],
   "source": [
    "dataset = json_deserialize(\n",
    "    data,\n",
    "    data_filter={\n",
    "        ComponentType.node: None,  # nodes in a row-based data format\n",
    "        ComponentType.source: [\"id\", \"node\", \"status\", \"u_ref\", \"sk\"],  # only specific attributes\n",
    "        ComponentType.sym_load: ComponentAttributeFilterOptions.everything,  # all attributes as columns\n",
    "        ComponentType.line: ComponentAttributeFilterOptions.relevant,  # only attributes that are not null/nan\n",
    "    },\n",
    ")\n",
    "\n",
    "print(\"components:\", list(dataset.keys()))\n",
    "print(\"node attributes:\", list(dataset[ComponentType.node].dtype.names))\n",
    "print(\"source attributes:\", list(dataset[ComponentType.source].keys()))\n",
    "print(\"sym_load attributes:\", list(dataset[ComponentType.sym_load].keys()))\n",
    "print(\"line attributes:\", list(dataset[ComponentType.line].keys()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A columnar dataset can also be serialized again, as one would expect."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"version\": \"1.0\",\n",
      "  \"type\": \"sym_output\",\n",
      "  \"is_batch\": false,\n",
      "  \"attributes\": {},\n",
      "  \"data\": {\n",
      "    \"node\": [\n",
      "      {\"id\": 1, \"energized\": 1, \"u_pu\": 1.030000000001025, \"u\": 10815.00000001077, \"u_angle\": -2.530316910142707e-14, \"p\": 2408997.839438867, \"q\": -2863495.364674167},\n",
      "      {\"id\": 2, \"energized\": 1, \"u_pu\": 1.029996969815606, \"u\": 10814.96818306386, \"u_angle\": -0.004397999804754745, \"p\": -1009999.99999997, \"q\": -210000.0000000655},\n",
      "      {\"id\": 3, \"energized\": 1, \"u_pu\": 1.029483905569345, \"u\": 10809.58100847812, \"u_angle\": -0.006839956175380238, \"p\": -1019999.999999999, \"q\": -219999.9999999689}\n",
      "    ],\n",
      "    \"line\": [\n",
      "      {\"id\": 4, \"energized\": 1, \"loading\": 0.3995319091937107, \"p_from\": 2408997.839438867, \"q_from\": -2863495.364674167, \"i_from\": 199.7659545968554, \"s_from\": 3742041.7279784, \"p_to\": -2252625.764367544, \"q_to\": 1403928.536947823, \"i_to\": 141.6984332838951, \"s_to\": 2654305.591138465},\n",
      "      {\"id\": 5, \"energized\": 1, \"loading\": 0.1977047433812966, \"p_from\": 1242625.764367574, \"q_from\": -1613928.536947772, \"i_from\": 108.7376088597131, \"s_from\": 2036880.976553238, \"p_to\": -1019999.999999999, \"q_to\": -219999.9999999689, \"i_to\": 55.73199226981006, \"s_to\": 1043455.796859639}\n",
      "    ],\n",
      "    \"source\": [\n",
      "      {\"id\": 15, \"energized\": 1, \"p\": -7836685.751732428, \"q\": -105348495.343833, \"i\": 5639.485452974508, \"s\": 105639571.7275539, \"pf\": -0.07418324046166491},\n",
      "      {\"id\": 16, \"energized\": 1, \"p\": 10248883.05839198, \"q\": 102488830.5811954, \"i\": 5498.573991718361, \"s\": 102999999.9895415, \"pf\": 0.09950371902361781},\n",
      "      {\"id\": 17, \"energized\": 1, \"p\": -0.001807829591278542, \"q\": -0.0104081191056169, \"i\": 5.639485452974507e-07, \"s\": 0.01056395717275539, \"pf\": -0.1711318553942043}\n",
      "    ],\n",
      "    \"sym_load\": [\n",
      "      {\"id\": 7, \"energized\": 1, \"p\": 1010000, \"q\": 210000, \"i\": 55.07135393955915, \"s\": 1031600.697944704, \"pf\": 0.979060989404389},\n",
      "      {\"id\": 8, \"energized\": 1, \"p\": 1020000, \"q\": 220000, \"i\": 55.73199226981046, \"s\": 1043455.796859647, \"pf\": 0.9775210440823288}\n",
      "    ]\n",
      "  }\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "serialized_output = json_serialize(output)\n",
    "\n",
    "print(serialized_output)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}